# -*- coding: utf-8 -*-
import datetime
import socket
from urllib.parse import urljoin

from scrapy import Spider, Request
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose

from mansite.items import AverItem


class AverSpider(Spider):
    name = 'aver'
    allowed_domains = 'http://m.nanrenvip.cc'
    start_urls = [
        'http://m.nanrenvip.cc/nvyouku/'
    ]

    def parse(self, response):

        # 水平爬取
        next_selector = response.xpath('//div[@class="page"]/span[@class="right"]/a[1]').extract()
        self.log(next_selector)
        next_urls = response.xpath('//div[@class="page"]/span[@class="right"]/a[1]/@href').extract()
        self.log('the next aver url %s' % next_urls)

        next_parse_url = urljoin(response.url, next_urls[0])
        if response.url != next_parse_url:
            self.log('next parse url: %s' % next_parse_url)
            yield Request(url=next_parse_url, dont_filter=True)

        # for url in next_urls:
        #     # 异步启动遍历下一页
        #     nextUrl = urljoin(response.url, url)
        #     self.log('next url %s' % nextUrl)
        #     yield Request(url=nextUrl, callback=self.parse, dont_filter=True)

        selectors = response.xpath('//li[@class="book-li"]')
        self.log('================ selector len %d' % len(selectors))
        for selector in selectors:
            yield self.parse_item(selector, response)

    # 解析selector中的Item
    def parse_item(self, selector, response):
        # Item 装载器与管理字段
        # 声明装载器
        ld = ItemLoader(item=AverItem(), selector=selector)

        # 使用装载器管理字段
        ld.add_xpath('starName', './/h4[@class="book-title"]/text()', MapCompose(lambda i: i.strip()))
        ld.add_xpath('intro', './/p[@class="book-desc"]/text()', MapCompose(lambda i: i.strip()))
        ld.add_xpath('avatar', './/img/@data-echo', MapCompose(lambda i: urljoin(response.url, i)))

        # 系统参数
        ld.add_value('spiderUrl', response.url)
        ld.add_value('timestamp', datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        ld.add_value('spiderName', self.name)
        ld.add_value('spiderProject', self.settings.get('BOT_NAME'))
        ld.add_value('hostname', socket.gethostname())

        item = ld.load_item()
        # self.log('return item %s' % item)
        return item
